﻿using System;
using System.Collections.Generic;
using System.Net;
using System.Threading;

namespace Jayden.Dll.Web.Crawler
{
	public class WebCrawler
	{
		private WebCrawlerProject m_Project;
		private List<Thread> m_Threads = new List<Thread>();
		private bool m_Running = false;
		private bool m_Stopping = false;

		public WebCrawler(WebCrawlerProject project)
		{
			if (project == null)
				throw new NullReferenceException("project cannot be null.");
			m_Project = project;
		}

		public WebCrawlerProject Project { get { return m_Project; } }

		public void Start(int threads = 1)
		{
			lock (m_Threads)
			{
				if (m_Running)
					throw new Exception("crawler already running.");
				m_Running = true;
				for (int c = 0; c < threads; c++)
				{
					Thread thread = new Thread(Crawl);
					m_Threads.Add(thread);
					thread.Start();
				}
			}
		}

		public void Stop()
		{
			lock (m_Threads)
			{
				if (!m_Running)
					throw new Exception("crawler not running.");
				if(m_Stopping)
					throw new Exception("crawler already stopping.");
				m_Stopping = true;
				foreach (Thread thread in m_Threads)
				{
					thread.Interrupt();
					thread.Join();
				}
				m_Running = false;
				m_Stopping = false;
			}
		}

		private void Crawl()
		{
			WebClient client = new WebClient();
			WebCrawlerUri uri = PopUri();
			while (uri != null && !m_Stopping)
			{
				try
				{
					byte[] data = client.DownloadData(uri.Uri.ToString());
					string contentType = client.ResponseHeaders["content-type"];
					ProcessData(uri, contentType, data);
				}
				catch (Exception e)
				{
					lock (m_Project.UrisFailed)
					{
						m_Project.UrisFailed.Add(uri);
					}
				}
				if(!m_Stopping)
					uri = PopUri();
			}
		}

		protected virtual void ProcessData(WebCrawlerUri uri, string contentType, byte[] data)
		{
			foreach (DataProcessor processor in m_Project.DataProcessors)
				if(processor.Validator == null || processor.Validator.IsValidUri(uri))
					processor.ProcessData(uri, contentType, data, this);
		}

		private WebCrawlerUri PopUri()
		{
			try
			{
				WebCrawlerUri uri = null;
				lock (m_Project.UrisLeft)
				{
					while (m_Project.UrisLeft.Count == 0)
						Monitor.Wait(m_Project.UrisLeft);
					if (m_Project.UrisLeft.Count > 0)
					{
						if (m_Project.NeedToSortUri && m_Project.UriSorter != null)
						{
							m_Project.UrisLeft.Sort(m_Project.UriSorter);
							m_Project.NeedToSortUri = false;
						}
						uri = m_Project.UrisLeft[0];
						m_Project.UrisLeft.RemoveAt(0);
					}
				}
				if (uri != null)
				{
					lock (m_Project.UrisDone)
					{
						m_Project.UrisDone.Add(uri);
					}
				}
				return uri;
			}
			catch (ThreadInterruptedException exc)
			{
				return null;
			}
		}
	}
}
